#coding: UTF-8
class Parservice
  attr_accessor :lines, :blockWidth, :threshold, :html, :flag, :start, :end, :text, :indexDistribution

  def initialize flag, blockWidth, threshold
    @flag=flag
    @blockWidth=blockWidth
    @threshold=threshold
    @html=""
    @indexDistribution=Array.new
    @lines
    @threshold
    @flag
    @start
    @end
    @text=''

  end

  def preProcess()

    @html=@html.gsub(/<!DOCTYPE.*>/iu, '')
    #  @html=@html.gsub(/<!--.*-->/mu, '')
    @html=@html.gsub(/<script.*?>.*?<\/script>/miu, '')
    @html=@html.gsub(/<style.*?>.*?<\/style>/imu, '')
    @html=@html.gsub(/<.*?>/u, '')
    @html=@html.gsub(/&.{1,5};|&#.{1,5};/u, '')
  end

  def getText
    @lines=@html.split(/\n/u)
    0.upto(@lines.length - blockWidth-1) {
        |i|
      wordsNum=0
      i.upto(i+blockWidth-1) {
          |j|
        @lines[j]=@lines[j].gsub(/\s+/mu, '')
        wordsNum+=@lines[j].size
      }
      indexDistribution[i]=wordsNum

    }
    @start=-1; @end=-1;
    boolstart=false; boolend=false;

    0.upto(@indexDistribution.length-2) {
        |i|
      if @indexDistribution[i]>@threshold && !boolstart
        if @indexDistribution[i+1]!=0 || @indexDistribution[i+2]!=0 || @indexDistribution[i+3]!=0
          boolstart=true
          @start=i
          next
        end
      end
      if boolstart
        if @indexDistribution[i]==0 || @indexDistribution[i+1]==0
          @end=i
          boolend=true

        end
      end
      temp=''
      if boolend
        @start.upto(@end) {
            |ii|
          if @lines[ii].size<5
            next
            temp+=@lines[ii] +"\n"
          end
          @text=@text+temp
          boolstart=false;
          boolend=false;

        }
      end

    }

    return @text
  end
  def parse _html, _flag=false
    @flag=_flag
    @html=_html
    preProcess()
    return getText();

  end
end




parser=Parservice.new(false, 3, 86)
_html=''
file=File.open("test.txt", "r:UTF-8") { |file|
  while line=file.gets do
    _html+=line
  end
}
#puts _html
puts parser.parse(_html)
